{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch：优化模块optim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "到目前为止，我们已经通过手动改变包含可学习参数的张量来更新模型的权重。对于随机梯度下降(SGD/stochastic gradient descent)等简单的优化算法来说，这不是一个很大的负担，但在实践中，我们经常使用AdaGrad、RMSProp、Adam等更复杂的优化器来训练神经网络。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 716.6713256835938\n",
      "1 699.1139526367188\n",
      "2 682.10595703125\n",
      "3 665.6494140625\n",
      "4 649.6815185546875\n",
      "5 634.2293701171875\n",
      "6 619.2075805664062\n",
      "7 604.6422119140625\n",
      "8 590.4629516601562\n",
      "9 576.785400390625\n",
      "10 563.4769897460938\n",
      "11 550.5133056640625\n",
      "12 537.9463500976562\n",
      "13 525.7206420898438\n",
      "14 513.8168334960938\n",
      "15 502.2776184082031\n",
      "16 490.98291015625\n",
      "17 479.9399108886719\n",
      "18 469.2401428222656\n",
      "19 458.8514099121094\n",
      "20 448.75140380859375\n",
      "21 438.90313720703125\n",
      "22 429.31536865234375\n",
      "23 420.0145263671875\n",
      "24 410.9399719238281\n",
      "25 402.09063720703125\n",
      "26 393.49542236328125\n",
      "27 385.1069030761719\n",
      "28 376.9089050292969\n",
      "29 368.90533447265625\n",
      "30 361.0704040527344\n",
      "31 353.41680908203125\n",
      "32 345.9276123046875\n",
      "33 338.6028747558594\n",
      "34 331.4344787597656\n",
      "35 324.42156982421875\n",
      "36 317.5260314941406\n",
      "37 310.7529296875\n",
      "38 304.1239318847656\n",
      "39 297.6372375488281\n",
      "40 291.2694091796875\n",
      "41 285.0007019042969\n",
      "42 278.8291931152344\n",
      "43 272.7663269042969\n",
      "44 266.83917236328125\n",
      "45 261.0252380371094\n",
      "46 255.3201904296875\n",
      "47 249.71835327148438\n",
      "48 244.23878479003906\n",
      "49 238.8614044189453\n",
      "50 233.5711212158203\n",
      "51 228.38002014160156\n",
      "52 223.28091430664062\n",
      "53 218.2646484375\n",
      "54 213.34036254882812\n",
      "55 208.5015411376953\n",
      "56 203.7621612548828\n",
      "57 199.1066131591797\n",
      "58 194.54946899414062\n",
      "59 190.06854248046875\n",
      "60 185.6748809814453\n",
      "61 181.35276794433594\n",
      "62 177.10194396972656\n",
      "63 172.9361114501953\n",
      "64 168.8482666015625\n",
      "65 164.83782958984375\n",
      "66 160.90652465820312\n",
      "67 157.03762817382812\n",
      "68 153.23837280273438\n",
      "69 149.50732421875\n",
      "70 145.84471130371094\n",
      "71 142.253662109375\n",
      "72 138.73521423339844\n",
      "73 135.2784881591797\n",
      "74 131.88975524902344\n",
      "75 128.5672607421875\n",
      "76 125.29510498046875\n",
      "77 122.0846939086914\n",
      "78 118.93617248535156\n",
      "79 115.84098815917969\n",
      "80 112.80213165283203\n",
      "81 109.81822967529297\n",
      "82 106.89390563964844\n",
      "83 104.02215576171875\n",
      "84 101.21138000488281\n",
      "85 98.43939971923828\n",
      "86 95.72408294677734\n",
      "87 93.06179809570312\n",
      "88 90.45491027832031\n",
      "89 87.89348602294922\n",
      "90 85.38404846191406\n",
      "91 82.93260955810547\n",
      "92 80.5330810546875\n",
      "93 78.18292236328125\n",
      "94 75.88236999511719\n",
      "95 73.62340545654297\n",
      "96 71.40789031982422\n",
      "97 69.23896026611328\n",
      "98 67.11858367919922\n",
      "99 65.04373931884766\n",
      "100 63.01570129394531\n",
      "101 61.03660583496094\n",
      "102 59.101234436035156\n",
      "103 57.21232604980469\n",
      "104 55.36881637573242\n",
      "105 53.570030212402344\n",
      "106 51.81544876098633\n",
      "107 50.10824966430664\n",
      "108 48.442726135253906\n",
      "109 46.82433319091797\n",
      "110 45.249595642089844\n",
      "111 43.71765899658203\n",
      "112 42.228416442871094\n",
      "113 40.78321838378906\n",
      "114 39.376068115234375\n",
      "115 38.00786590576172\n",
      "116 36.679107666015625\n",
      "117 35.386558532714844\n",
      "118 34.13337326049805\n",
      "119 32.916683197021484\n",
      "120 31.73749542236328\n",
      "121 30.594558715820312\n",
      "122 29.485227584838867\n",
      "123 28.408519744873047\n",
      "124 27.366437911987305\n",
      "125 26.35499382019043\n",
      "126 25.375904083251953\n",
      "127 24.428443908691406\n",
      "128 23.50956153869629\n",
      "129 22.620986938476562\n",
      "130 21.76082420349121\n",
      "131 20.92909049987793\n",
      "132 20.124967575073242\n",
      "133 19.346731185913086\n",
      "134 18.595012664794922\n",
      "135 17.869184494018555\n",
      "136 17.168354034423828\n",
      "137 16.491779327392578\n",
      "138 15.838333129882812\n",
      "139 15.207640647888184\n",
      "140 14.600202560424805\n",
      "141 14.013981819152832\n",
      "142 13.448375701904297\n",
      "143 12.902846336364746\n",
      "144 12.376187324523926\n",
      "145 11.867444038391113\n",
      "146 11.376187324523926\n",
      "147 10.902517318725586\n",
      "148 10.445838928222656\n",
      "149 10.00594711303711\n",
      "150 9.581748962402344\n",
      "151 9.173506736755371\n",
      "152 8.780083656311035\n",
      "153 8.401399612426758\n",
      "154 8.037785530090332\n",
      "155 7.687758445739746\n",
      "156 7.3513360023498535\n",
      "157 7.028284072875977\n",
      "158 6.717506408691406\n",
      "159 6.419267654418945\n",
      "160 6.132995128631592\n",
      "161 5.857882976531982\n",
      "162 5.59421968460083\n",
      "163 5.341015815734863\n",
      "164 5.098249435424805\n",
      "165 4.865693092346191\n",
      "166 4.642749309539795\n",
      "167 4.42893648147583\n",
      "168 4.2240400314331055\n",
      "169 4.02790641784668\n",
      "170 3.8396332263946533\n",
      "171 3.6596412658691406\n",
      "172 3.487497329711914\n",
      "173 3.322821617126465\n",
      "174 3.1654491424560547\n",
      "175 3.014889717102051\n",
      "176 2.8710930347442627\n",
      "177 2.7335879802703857\n",
      "178 2.6024601459503174\n",
      "179 2.4771862030029297\n",
      "180 2.357525587081909\n",
      "181 2.2434868812561035\n",
      "182 2.134641408920288\n",
      "183 2.030977725982666\n",
      "184 1.9320343732833862\n",
      "185 1.8377909660339355\n",
      "186 1.7478982210159302\n",
      "187 1.6623440980911255\n",
      "188 1.581134557723999\n",
      "189 1.504106879234314\n",
      "190 1.4313021898269653\n",
      "191 1.3619554042816162\n",
      "192 1.2959707975387573\n",
      "193 1.233206868171692\n",
      "194 1.1734554767608643\n",
      "195 1.1166101694107056\n",
      "196 1.0625585317611694\n",
      "197 1.0111041069030762\n",
      "198 0.9621508121490479\n",
      "199 0.9156542420387268\n",
      "200 0.8713756203651428\n",
      "201 0.8293194770812988\n",
      "202 0.7892874479293823\n",
      "203 0.7512274384498596\n",
      "204 0.7150230407714844\n",
      "205 0.6806104779243469\n",
      "206 0.647813081741333\n",
      "207 0.616648256778717\n",
      "208 0.587048351764679\n",
      "209 0.5588365793228149\n",
      "210 0.5320267677307129\n",
      "211 0.5064975619316101\n",
      "212 0.4822295308113098\n",
      "213 0.459134578704834\n",
      "214 0.4371638596057892\n",
      "215 0.41626203060150146\n",
      "216 0.3963813781738281\n",
      "217 0.37745723128318787\n",
      "218 0.3594401776790619\n",
      "219 0.3423078656196594\n",
      "220 0.3259854316711426\n",
      "221 0.310468465089798\n",
      "222 0.2956823408603668\n",
      "223 0.28162217140197754\n",
      "224 0.2682228982448578\n",
      "225 0.2554720342159271\n",
      "226 0.24333561956882477\n",
      "227 0.2317744940519333\n",
      "228 0.22076517343521118\n",
      "229 0.21028397977352142\n",
      "230 0.2003045380115509\n",
      "231 0.19079457223415375\n",
      "232 0.18173521757125854\n",
      "233 0.1731090247631073\n",
      "234 0.16488540172576904\n",
      "235 0.15705277025699615\n",
      "236 0.14959248900413513\n",
      "237 0.14248354732990265\n",
      "238 0.1357116401195526\n",
      "239 0.1292600929737091\n",
      "240 0.12311400473117828\n",
      "241 0.11725577712059021\n",
      "242 0.11167347431182861\n",
      "243 0.10635402053594589\n",
      "244 0.10128546506166458\n",
      "245 0.09645535796880722\n",
      "246 0.09185019880533218\n",
      "247 0.08746115863323212\n",
      "248 0.08327942341566086\n",
      "249 0.07929503172636032\n",
      "250 0.07549756020307541\n",
      "251 0.07187774032354355\n",
      "252 0.06842928379774094\n",
      "253 0.06514148414134979\n",
      "254 0.06201234832406044\n",
      "255 0.059029072523117065\n",
      "256 0.056187909096479416\n",
      "257 0.05347936227917671\n",
      "258 0.050898756831884384\n",
      "259 0.04843844100832939\n",
      "260 0.04609586298465729\n",
      "261 0.04386350139975548\n",
      "262 0.04173758625984192\n",
      "263 0.039709486067295074\n",
      "264 0.03777747228741646\n",
      "265 0.035938408225774765\n",
      "266 0.034186434000730515\n",
      "267 0.03251603990793228\n",
      "268 0.030925655737519264\n",
      "269 0.029410775750875473\n",
      "270 0.027967944741249084\n",
      "271 0.026593824848532677\n",
      "272 0.025285519659519196\n",
      "273 0.024039365351200104\n",
      "274 0.022852888330817223\n",
      "275 0.02172262966632843\n",
      "276 0.02064650133252144\n",
      "277 0.019622283056378365\n",
      "278 0.01864750310778618\n",
      "279 0.01771944761276245\n",
      "280 0.01683596707880497\n",
      "281 0.015995077788829803\n",
      "282 0.015195227228105068\n",
      "283 0.014433449134230614\n",
      "284 0.013708911836147308\n",
      "285 0.01301953662186861\n",
      "286 0.012363762594759464\n",
      "287 0.011740063317120075\n",
      "288 0.01114639826118946\n",
      "289 0.010581998154520988\n",
      "290 0.010045338422060013\n",
      "291 0.009534911252558231\n",
      "292 0.009049643762409687\n",
      "293 0.008588168770074844\n",
      "294 0.00814924854785204\n",
      "295 0.007732338737696409\n",
      "296 0.00733606331050396\n",
      "297 0.006959410384297371\n",
      "298 0.006601471453905106\n",
      "299 0.006262024398893118\n",
      "300 0.005939683876931667\n",
      "301 0.00563324149698019\n",
      "302 0.0053422702476382256\n",
      "303 0.005065792240202427\n",
      "304 0.004803194664418697\n",
      "305 0.004554021172225475\n",
      "306 0.004317157901823521\n",
      "307 0.004092393908649683\n",
      "308 0.0038790267426520586\n",
      "309 0.0036763548851013184\n",
      "310 0.003484010696411133\n",
      "311 0.0033014181535691023\n",
      "312 0.0031281260307878256\n",
      "313 0.002963664475828409\n",
      "314 0.002807628596201539\n",
      "315 0.002659524092450738\n",
      "316 0.0025190534070134163\n",
      "317 0.00238579954020679\n",
      "318 0.0022594043985009193\n",
      "319 0.0021394581999629736\n",
      "320 0.002025812631472945\n",
      "321 0.0019179574446752667\n",
      "322 0.001815690309740603\n",
      "323 0.00171872868668288\n",
      "324 0.0016268659383058548\n",
      "325 0.0015397310489788651\n",
      "326 0.001457134261727333\n",
      "327 0.001378851244226098\n",
      "328 0.0013046851381659508\n",
      "329 0.0012344011338427663\n",
      "330 0.0011677759466692805\n",
      "331 0.001104662660509348\n",
      "332 0.0010448823450133204\n",
      "333 0.0009882751619443297\n",
      "334 0.0009346342412754893\n",
      "335 0.0008838974172249436\n",
      "336 0.0008357217302545905\n",
      "337 0.0007901733042672276\n",
      "338 0.0007470494601875544\n",
      "339 0.0007062272052280605\n",
      "340 0.000667582789901644\n",
      "341 0.0006309943273663521\n",
      "342 0.0005963535513728857\n",
      "343 0.0005635976558551192\n",
      "344 0.0005325825768522918\n",
      "345 0.0005032439948990941\n",
      "346 0.0004754836263600737\n",
      "347 0.0004492237640079111\n",
      "348 0.0004243789881002158\n",
      "349 0.0004008839896414429\n",
      "350 0.0003786478191614151\n",
      "351 0.00035762746119871736\n",
      "352 0.0003377451794221997\n",
      "353 0.0003189474518876523\n",
      "354 0.00030118299764581025\n",
      "355 0.00028437681612558663\n",
      "356 0.00026849229470826685\n",
      "357 0.00025347000337205827\n",
      "358 0.00023928817245177925\n",
      "359 0.00022586859995499253\n",
      "360 0.00021319380903150886\n",
      "361 0.00020122049318160862\n",
      "362 0.00018991073011420667\n",
      "363 0.00017920034588314593\n",
      "364 0.0001690990466158837\n",
      "365 0.00015956375864334404\n",
      "366 0.00015054948744364083\n",
      "367 0.00014203428872860968\n",
      "368 0.00013399416639003903\n",
      "369 0.00012640196655411273\n",
      "370 0.00011922958947252482\n",
      "371 0.00011246301437495276\n",
      "372 0.00010606001887936145\n",
      "373 0.00010002687486121431\n",
      "374 9.433152445126325e-05\n",
      "375 8.895232167560607e-05\n",
      "376 8.387737034354359e-05\n",
      "377 7.908733823569492e-05\n",
      "378 7.456273306161165e-05\n",
      "379 7.029485277598724e-05\n",
      "380 6.626943650189787e-05\n",
      "381 6.246636621654034e-05\n",
      "382 5.8883299061562866e-05\n",
      "383 5.5498250731034204e-05\n",
      "384 5.2311854233266786e-05\n",
      "385 4.92967119498644e-05\n",
      "386 4.645852095563896e-05\n",
      "387 4.378373341751285e-05\n",
      "388 4.1255028918385506e-05\n",
      "389 3.887221828335896e-05\n",
      "390 3.662817471195012e-05\n",
      "391 3.450921212788671e-05\n",
      "392 3.251227826694958e-05\n",
      "393 3.0627790692960843e-05\n",
      "394 2.8850994567619637e-05\n",
      "395 2.7177096853847615e-05\n",
      "396 2.560072607593611e-05\n",
      "397 2.4111761376843788e-05\n",
      "398 2.2708865799359046e-05\n",
      "399 2.1387681044870988e-05\n",
      "400 2.0141786080785096e-05\n",
      "401 1.896677895274479e-05\n",
      "402 1.7860580555861816e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "403 1.6819178199511953e-05\n",
      "404 1.5834699297556654e-05\n",
      "405 1.4908871889929287e-05\n",
      "406 1.403620535711525e-05\n",
      "407 1.3213968486525118e-05\n",
      "408 1.2441327271517366e-05\n",
      "409 1.1709991667885333e-05\n",
      "410 1.1022725630027708e-05\n",
      "411 1.0375159945397172e-05\n",
      "412 9.764738933881745e-06\n",
      "413 9.19139347388409e-06\n",
      "414 8.651054486108478e-06\n",
      "415 8.141207217704505e-06\n",
      "416 7.660773007955868e-06\n",
      "417 7.209368050098419e-06\n",
      "418 6.7823675635736436e-06\n",
      "419 6.383131676557241e-06\n",
      "420 6.0044590100005735e-06\n",
      "421 5.6500002756365575e-06\n",
      "422 5.315369435265893e-06\n",
      "423 5.001465979148634e-06\n",
      "424 4.704780621977989e-06\n",
      "425 4.4251778490433935e-06\n",
      "426 4.162698132859077e-06\n",
      "427 3.915390152542386e-06\n",
      "428 3.6823034861299675e-06\n",
      "429 3.4632396364031592e-06\n",
      "430 3.257152457081247e-06\n",
      "431 3.063304120587418e-06\n",
      "432 2.8805563943024026e-06\n",
      "433 2.7085734473075718e-06\n",
      "434 2.5467847990512382e-06\n",
      "435 2.3950433387653902e-06\n",
      "436 2.2519654976349557e-06\n",
      "437 2.1172998003748944e-06\n",
      "438 1.9906267425540136e-06\n",
      "439 1.8712928522290895e-06\n",
      "440 1.7586095282240422e-06\n",
      "441 1.6535102531634038e-06\n",
      "442 1.55422935677052e-06\n",
      "443 1.4605301430492545e-06\n",
      "444 1.3725785947826807e-06\n",
      "445 1.2899770354124485e-06\n",
      "446 1.212283450513496e-06\n",
      "447 1.139315145337605e-06\n",
      "448 1.0706903594837058e-06\n",
      "449 1.005679678200977e-06\n",
      "450 9.449801154914894e-07\n",
      "451 8.879558208718663e-07\n",
      "452 8.341875172845903e-07\n",
      "453 7.837007842681487e-07\n",
      "454 7.360869744843512e-07\n",
      "455 6.913775791872467e-07\n",
      "456 6.49380353934248e-07\n",
      "457 6.099344886933977e-07\n",
      "458 5.728614382860542e-07\n",
      "459 5.380413199418399e-07\n",
      "460 5.050566187492223e-07\n",
      "461 4.742171029192832e-07\n",
      "462 4.4543693888954294e-07\n",
      "463 4.1800015537774016e-07\n",
      "464 3.9250454619832453e-07\n",
      "465 3.6841163364442764e-07\n",
      "466 3.457618333868595e-07\n",
      "467 3.2457757015436073e-07\n",
      "468 3.0484133617392217e-07\n",
      "469 2.8609591140593693e-07\n",
      "470 2.6835004973690957e-07\n",
      "471 2.51728351940983e-07\n",
      "472 2.3636401635940274e-07\n",
      "473 2.2170614499827934e-07\n",
      "474 2.0809223144624411e-07\n",
      "475 1.9522661887094728e-07\n",
      "476 1.8318024785912712e-07\n",
      "477 1.7185917045026144e-07\n",
      "478 1.611434186088445e-07\n",
      "479 1.511382237140424e-07\n",
      "480 1.4170417728109896e-07\n",
      "481 1.3295392875534162e-07\n",
      "482 1.2457300613277766e-07\n",
      "483 1.1686304191016461e-07\n",
      "484 1.0960235385937267e-07\n",
      "485 1.027277534149107e-07\n",
      "486 9.625512120692292e-08\n",
      "487 9.022432578831285e-08\n",
      "488 8.460418854383533e-08\n",
      "489 7.931988221798747e-08\n",
      "490 7.431430759652358e-08\n",
      "491 6.970442001374977e-08\n",
      "492 6.538103747288915e-08\n",
      "493 6.117539896877133e-08\n",
      "494 5.7345715731571545e-08\n",
      "495 5.3751357143028144e-08\n",
      "496 5.032304173369084e-08\n",
      "497 4.7125148228133185e-08\n",
      "498 4.42057519478567e-08\n",
      "499 4.142384213423611e-08\n"
     ]
    }
   ],
   "source": [
    "# 可运行代码见本文件夹中的 two_layer_net_optim.py\n",
    "import torch\n",
    "\n",
    "# N是批大小；D是输入维度\n",
    "# H是隐藏层维度；D_out是输出维度\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# 产生随机输入和输出张量\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# 使用nn包定义模型和损失函数\n",
    "model = torch.nn.Sequential(\n",
    "          torch.nn.Linear(D_in, H),\n",
    "          torch.nn.ReLU(),\n",
    "          torch.nn.Linear(H, D_out),\n",
    "        )\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "# 使用optim包定义优化器（Optimizer）。Optimizer将会为我们更新模型的权重。\n",
    "# 这里我们使用Adam优化方法；optim包还包含了许多别的优化算法。\n",
    "# Adam构造函数的第一个参数告诉优化器应该更新哪些张量。\n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for t in range(500):\n",
    "\n",
    "    # 前向传播：通过像模型输入x计算预测的y\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # 计算并打印loss\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "    \n",
    "    # 在反向传播之前，使用optimizer将它要更新的所有张量的梯度清零(这些张量是模型可学习的权重)\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    # 反向传播：根据模型的参数计算loss的梯度\n",
    "    loss.backward()\n",
    "\n",
    "    # 调用Optimizer的step函数使它所有参数更新\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (Spyder)",
   "language": "python3",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "227.797px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
